package tokenizer;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Scanner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Tokenizer {
	List<String> filter = Arrays.asList(new String[]{
			"bool","boolean","byte","char","int","long","short","float","double","private","protected",
			"public","abstract","final","native","static","try","catch","final","throw","break","case","continue",
			"default","do","while","for","switch","if","else","elseif","endif","ifdef","class","extends","import",
			"new","return","void","String","string","super","true","false","null","object"});
	Token[] tokens;
	public void readFile(File file) throws FileNotFoundException {
		Map<String, Integer> tokenMap = new HashMap<String, Integer>();
		Pattern tokenPattern = Pattern.compile("\\W*([a-zA-Z]\\w*)\\W*");
		Scanner scanner = new Scanner(file);
		while (scanner.hasNext()) {
			String next = scanner.next();
			Matcher matcher = tokenPattern.matcher(next);
			if(matcher.matches()){
				next = matcher.group(1);
			}else{
				continue;
			}
			if(filter.contains(next)){
				continue;
			}
			if (tokenMap.containsKey(next)) {
				int nextv = ((Integer) tokenMap.get(next)) + 1;
				tokenMap.put(next, nextv);
			} else {
				tokenMap.put(next, 1);
			}
		}
		tokens = new Token[tokenMap.size()];
		int index=0;
		for (Entry<String, Integer> entry : tokenMap.entrySet()){
			tokens[index++]=new Token(entry.getKey(),entry.getValue());	
		}
		Arrays.sort(tokens);
	}
	public String toString(){
		StringBuffer sb = new StringBuffer("");
		for(Token token:tokens){
			sb.append(token.toString());
		}
		return sb.toString();
	}
}
